{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:00:33.234794Z",
     "start_time": "2025-01-23T08:00:30.839118Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:06.194795Z",
     "iopub.status.busy": "2025-01-23T08:17:06.194511Z",
     "iopub.status.idle": "2025-01-23T08:17:08.288087Z",
     "shell.execute_reply": "2025-01-23T08:17:08.287432Z",
     "shell.execute_reply.started": "2025-01-23T08:17:06.194772Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=10, micro=14, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.5.1+cu124\n",
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-02T03:33:53.350580400Z",
     "start_time": "2024-05-02T03:33:52.447226500Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.289369Z",
     "iopub.status.busy": "2025-01-23T08:17:08.289041Z",
     "iopub.status.idle": "2025-01-23T08:17:08.292186Z",
     "shell.execute_reply": "2025-01-23T08:17:08.291580Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.289346Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:00:56.509250Z",
     "start_time": "2025-01-23T08:00:56.504813Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.292951Z",
     "iopub.status.busy": "2025-01-23T08:17:08.292769Z",
     "iopub.status.idle": "2025-01-23T08:17:08.297467Z",
     "shell.execute_reply": "2025-01-23T08:17:08.296818Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.292931Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length 1115394\n",
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You\n"
     ]
    }
   ],
   "source": [
    "# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
    "#文件已经下载好了\n",
    "with open(\"./shakespeare.txt\", \"r\", encoding=\"utf8\") as file:\n",
    "    text = file.read()\n",
    "\n",
    "print(\"length\", len(text))\n",
    "print(text[0:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构造字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:01:28.777548Z",
     "start_time": "2025-01-23T08:01:28.769444Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.298036Z",
     "iopub.status.busy": "2025-01-23T08:17:08.297868Z",
     "iopub.status.idle": "2025-01-23T08:17:08.318373Z",
     "shell.execute_reply": "2025-01-23T08:17:08.317661Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.298017Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n",
      "['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n"
     ]
    }
   ],
   "source": [
    "# 1. generate vocab\n",
    "# 2. build mapping char->id\n",
    "# 3. data -> id_data  把数据都转为id\n",
    "# 4. a b c d [EOS] -> [BOS] b c d  预测下一个字符生成的模型，也就是输入是a，输出就是b\n",
    "\n",
    "#去重，留下独立字符，并排序（排序是为了好看）\n",
    "vocab = sorted(set(text))\n",
    "print(len(vocab))\n",
    "print(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:01:36.496656Z",
     "start_time": "2025-01-23T08:01:36.493637Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.320835Z",
     "iopub.status.busy": "2025-01-23T08:17:08.320398Z",
     "iopub.status.idle": "2025-01-23T08:17:08.324538Z",
     "shell.execute_reply": "2025-01-23T08:17:08.323832Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.320803Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 h\n",
      "1 o\n",
      "2 w\n"
     ]
    }
   ],
   "source": [
    "for idx,char in enumerate(['h','o','w']):\n",
    "    print(idx,char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:01:43.290659Z",
     "start_time": "2025-01-23T08:01:43.288068Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.325508Z",
     "iopub.status.busy": "2025-01-23T08:17:08.325251Z",
     "iopub.status.idle": "2025-01-23T08:17:08.328898Z",
     "shell.execute_reply": "2025-01-23T08:17:08.328318Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.325470Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n"
     ]
    }
   ],
   "source": [
    "#每个字符都编好号，enumerate对每一个位置编号，生成的是列表中是元组，下面字典生成式\n",
    "char2idx = {char:idx for idx, char in enumerate(vocab)}\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:02:04.850128Z",
     "start_time": "2025-01-23T08:02:04.847295Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.330015Z",
     "iopub.status.busy": "2025-01-23T08:17:08.329574Z",
     "iopub.status.idle": "2025-01-23T08:17:08.333345Z",
     "shell.execute_reply": "2025-01-23T08:17:08.332744Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.329984Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n",
      " 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n",
      " 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n",
      " 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n"
     ]
    }
   ],
   "source": [
    "# 把vocab从列表变为ndarray\n",
    "idx2char = np.array(vocab)\n",
    "print(idx2char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:03:03.944020Z",
     "start_time": "2025-01-23T08:03:03.874617Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.334411Z",
     "iopub.status.busy": "2025-01-23T08:17:08.334017Z",
     "iopub.status.idle": "2025-01-23T08:17:08.451298Z",
     "shell.execute_reply": "2025-01-23T08:17:08.450748Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.334380Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1115394,)\n",
      "1115394\n",
      "[18 47 56 57 58  1 15 47 58 47]\n",
      "First Citi\n"
     ]
    }
   ],
   "source": [
    "#把字符都转换为id\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(text_as_int.shape)\n",
    "print(len(text_as_int))\n",
    "print(text_as_int[0:10])\n",
    "print(text[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:03:33.490881Z",
     "start_time": "2025-01-23T08:03:33.487237Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.452389Z",
     "iopub.status.busy": "2025-01-23T08:17:08.451936Z",
     "iopub.status.idle": "2025-01-23T08:17:08.456739Z",
     "shell.execute_reply": "2025-01-23T08:17:08.456178Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.452357Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11043"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1115394//101"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把莎士比亚文集分成一个一个的样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:09:37.251399Z",
     "start_time": "2025-01-23T08:09:37.247840Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.457842Z",
     "iopub.status.busy": "2025-01-23T08:17:08.457436Z",
     "iopub.status.idle": "2025-01-23T08:17:08.464781Z",
     "shell.execute_reply": "2025-01-23T08:17:08.464218Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.457810Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "    #text_as_int是字符的id列表，seq_length是每个样本的长度\n",
    "    def __init__(self, text_as_int, seq_length):\n",
    "        self.sub_len = seq_length + 1 #一个样本的长度\n",
    "        self.text_as_int = text_as_int\n",
    "        self.num_seq = len(text_as_int) // self.sub_len #样本的个数\n",
    "        \n",
    "    def __getitem__(self, index):#index是样本的索引，返回的是一个样本，比如第一个，就是0-100的字符,总计101个字符\n",
    "        return self.text_as_int[index * self.sub_len: (index + 1) * self.sub_len]\n",
    "    \n",
    "    def __len__(self): #返回样本的个数\n",
    "        return self.num_seq\n",
    "\n",
    "#batch是一个列表，列表中的每一个元素是一个样本，有101个字符，前100个是输入，后100个是输出\n",
    "def collat_fct(batch):\n",
    "    src_list = [] #输入\n",
    "    trg_list = [] #输出\n",
    "    for part in batch:\n",
    "        src_list.append(part[:-1]) #输入\n",
    "        trg_list.append(part[1:]) #输出\n",
    "        \n",
    "    src_list = np.array(src_list) #把列表转换为ndarray\n",
    "    trg_list = np.array(trg_list) #把列表转换为ndarray\n",
    "    return torch.Tensor(src_list).to(dtype=torch.int64), torch.Tensor(trg_list).to(dtype=torch.int64) #返回的是一个元组，元组中的每一个元素是一个torch.Tensor\n",
    "        \n",
    "#每个样本的长度是101，也就是100个字符+1个结束符\n",
    "train_ds = CharDataset(text_as_int, 100)\n",
    "train_dl = DataLoader(train_ds, batch_size=64, shuffle=True, collate_fn=collat_fct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:09:39.865440Z",
     "start_time": "2025-01-23T08:09:39.859267Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.465951Z",
     "iopub.status.busy": "2025-01-23T08:17:08.465498Z",
     "iopub.status.idle": "2025-01-23T08:17:08.471236Z",
     "shell.execute_reply": "2025-01-23T08:17:08.470746Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.465920Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 100])\n",
      "torch.Size([64, 100])\n"
     ]
    }
   ],
   "source": [
    "for datas, labels in train_dl:\n",
    "    print(datas.shape)\n",
    "    print(labels.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:13:43.092169Z",
     "start_time": "2025-01-23T08:13:43.083620Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.472221Z",
     "iopub.status.busy": "2025-01-23T08:17:08.471840Z",
     "iopub.status.idle": "2025-01-23T08:17:08.487855Z",
     "shell.execute_reply": "2025-01-23T08:17:08.487375Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.472190Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================== 一层单向 RNN ===================================\n",
      "            embedding.weight            paramerters num: 16640\n",
      "            rnn.weight_ih_l0            paramerters num: 262144\n",
      "            rnn.weight_hh_l0            paramerters num: 1048576\n",
      "             rnn.bias_ih_l0             paramerters num: 1024\n",
      "             rnn.bias_hh_l0             paramerters num: 1024\n",
      "               fc.weight                paramerters num: 66560\n",
      "                fc.bias                 paramerters num: 65\n"
     ]
    }
   ],
   "source": [
    "class CharRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024):\n",
    "        super(CharRNN, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        #batch_first=True,输入的数据格式是(batch_size, seq_len, embedding_dim)\n",
    "        self.rnn = nn.RNN(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        \n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.embedding(x) #(batch_size, seq_len) -> (batch_size, seq_len, embedding_dim) (64, 100, 256)\n",
    "        #这里和02的差异是没有只拿最后一个输出，而是把所有的输出都拿出来了\n",
    "        #(batch_size, seq_len, embedding_dim)->(batch_size, seq_len, hidden_dim)(64, 100, 1024)\n",
    "        output, hidden = self.rnn(x, hidden)\n",
    "        x = self.fc(output) #[bs, seq_len, hidden_dim]--->[bs, seq_len, vocab_size] (64, 100,65)\n",
    "        return x, hidden #x的shape是(batch_size, seq_len, vocab_size)\n",
    "    \n",
    "    \n",
    "vocab_size = len(vocab)\n",
    "\n",
    "print(\"{:=^80}\".format(\" 一层单向 RNN \"))       \n",
    "for key, value in CharRNN(vocab_size).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:15:23.555522Z",
     "start_time": "2025-01-23T08:15:23.551754Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.488861Z",
     "iopub.status.busy": "2025-01-23T08:17:08.488421Z",
     "iopub.status.idle": "2025-01-23T08:17:08.493065Z",
     "shell.execute_reply": "2025-01-23T08:17:08.492525Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.488829Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16640\n",
      "262144\n",
      "1048576\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "66560"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(65*256)\n",
    "print(256*1024)\n",
    "print(1024*1024)\n",
    "1024*65"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:27:35.393573400Z",
     "start_time": "2024-07-29T07:27:35.327680900Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.496358Z",
     "iopub.status.busy": "2025-01-23T08:17:08.496008Z",
     "iopub.status.idle": "2025-01-23T08:17:08.520922Z",
     "shell.execute_reply": "2025-01-23T08:17:08.520382Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.496327Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 100])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 100, 65])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_inputs = torch.randint(0, vocab_size, (3, 100))\n",
    "print(sample_inputs.shape)\n",
    "model = CharRNN(vocab_size)\n",
    "output=model(sample_inputs)\n",
    "output[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:57:28.992008300Z",
     "start_time": "2024-07-29T07:57:28.981317Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.521739Z",
     "iopub.status.busy": "2025-01-23T08:17:08.521557Z",
     "iopub.status.idle": "2025-01-23T08:17:08.525010Z",
     "shell.execute_reply": "2025-01-23T08:17:08.524573Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.521720Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([300, 65])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output[0].reshape(-1, vocab_size).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:26:37.116153100Z",
     "start_time": "2024-07-29T07:26:37.108972300Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.525644Z",
     "iopub.status.busy": "2025-01-23T08:17:08.525470Z",
     "iopub.status.idle": "2025-01-23T08:17:08.528873Z",
     "shell.execute_reply": "2025-01-23T08:17:08.528260Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.525625Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66560"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "1024*65"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:28:19.937860300Z",
     "start_time": "2024-07-29T07:28:19.932860Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.529522Z",
     "iopub.status.busy": "2025-01-23T08:17:08.529347Z",
     "iopub.status.idle": "2025-01-23T08:17:08.534592Z",
     "shell.execute_reply": "2025-01-23T08:17:08.534155Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.529503Z"
    }
   },
   "outputs": [],
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:16:19.788186Z",
     "start_time": "2025-01-23T08:16:18.860450Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:08.535220Z",
     "iopub.status.busy": "2025-01-23T08:17:08.535056Z",
     "iopub.status.idle": "2025-01-23T08:17:09.444746Z",
     "shell.execute_reply": "2025-01-23T08:17:09.444183Z",
     "shell.execute_reply.started": "2025-01-23T08:17:08.535201Z"
    }
   },
   "outputs": [],
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=None,\n",
    "    stateful=False      # 想用stateful，batch里的数据就必须连续，不能打乱\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    hidden = None\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算,如果数据集打乱了，stateful=False，hidden就要清空\n",
    "                # 如果数据集没有打乱，stateful=True，hidden就不需要清空\n",
    "                logits, hidden = model(datas, hidden=hidden if stateful else None)\n",
    "                # 计算损失,交叉熵损失第一个参数要是二阶张量，第二个参数要是一阶张量，所以要reshape\n",
    "                loss = loss_fct(logits.reshape(-1, vocab_size), labels.reshape(-1))\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "   \n",
    "                # 保存模型权重 save model checkpoint\n",
    "                if save_ckpt_callback is not None:\n",
    "                    save_ckpt_callback(global_step, model.state_dict(), metric=-loss)\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = CharRNN(vocab_size=vocab_size)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失 \n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "\n",
    "# save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/text_generation\", save_step=1000, save_best_only=True)\n",
    "\n",
    "\n",
    "model = model.to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:17:09.445822Z",
     "iopub.status.busy": "2025-01-23T08:17:09.445359Z",
     "iopub.status.idle": "2025-01-23T08:19:08.356217Z",
     "shell.execute_reply": "2025-01-23T08:19:08.355733Z",
     "shell.execute_reply.started": "2025-01-23T08:17:09.445799Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 17300/17300 [01:58<00:00, 145.49it/s, epoch=99]\n"
     ]
    }
   ],
   "source": [
    "record = training(\n",
    "    model,\n",
    "    train_dl,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-23T08:19:08.357145Z",
     "iopub.status.busy": "2025-01-23T08:19:08.356810Z",
     "iopub.status.idle": "2025-01-23T08:19:08.455159Z",
     "shell.execute_reply": "2025-01-23T08:19:08.454531Z",
     "shell.execute_reply.started": "2025-01-23T08:19:08.357124Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGdCAYAAADXIOPgAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWP1JREFUeJzt3Xl4U1X+BvA3SdN0Tfd9o1D2tkBZSgFZZAcVXNBBRlBxhxkQRx0cZxQdhZ8boijquOCoiKKCo6JQloJA2Vt2SsvSltKFtrTpmqbN+f2R5LahLdgFckvez/PwhNzcm5x8Cenbc885VyGEECAiIiKSGaWtG0BERETUFIYUIiIikiWGFCIiIpIlhhQiIiKSJYYUIiIikiWGFCIiIpIlhhQiIiKSJYYUIiIikiUHWzfgjzAajbhw4QLc3d2hUChs3RwiIiL6A4QQKCsrQ3BwMJTKlveLdIiQcuHCBYSFhdm6GURERNQK2dnZCA0NbfFxHSKkuLu7AzC9Sa1W227PazAYsHHjRowbNw5qtbrdnrejYR1YA4A1sGAdWAOANbBoax10Oh3CwsKkn+Mt1SFCiuUUj1arbfeQ4uLiAq1Wa/cfQnuvA2vAGliwDqwBwBpYtFcdWjtUgwNniYiISJYYUoiIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWGFKIiIhIluw6pHy2KxPfn1UiLa/M1k0hIiKiy9h1SFl/NA/b85TIvlRl66YQERHRZew6pCjNl442CmHjlhAREdHl7DqkKMy3RmYUIiIi2bHvkGJOKYI9KURERLJj5yHFlFKYUYiIiOTHrkOK0tKTYttmEBERURPsOqTUj0lhTCEiIpIbuw4pSp7uISIiki27Din1Y1KYUoiIiOTGzkOK6ZZTkImIiOTHrkNK/cBZphQiIiK5seuQooBlxVkbN4SIiIgase+QIi3mZtt2EBERUWMMKeDAWSIiIjlqU0hZsmQJFAoF5s+ff8X91qxZgx49esDJyQkxMTFYv359W1623UhTkG3cDiIiImqs1SFl3759+PDDDxEbG3vF/Xbt2oXp06dj9uzZSElJwdSpUzF16lQcPXq0tS/dbngVZCIiIvlqVUgpLy/HjBkz8J///AdeXl5X3HfZsmWYMGECnn76afTs2RMvv/wy4uLisHz58lY1+FrgwFkiIiL5cWjNQXPmzMHkyZMxZswY/Pvf/77ivsnJyViwYIHVtvHjx2PdunXNHqPX66HX66X7Op0OAGAwGGAwGFrT5CYpzCd66mrr2vV5OxrLe2cNWIOGt/aKdWANANbAoq11aGv9WhxSVq9ejYMHD2Lfvn1/aP+8vDwEBARYbQsICEBeXl6zxyxevBiLFi1qtH3jxo1wcXFpWYOvoCBfCUCJ4ydOYH3J8XZ73o4qMTHR1k2wOdaANbBgHVgDgDWwaG0dKisr2/S6LQop2dnZmDdvHhITE+Hk5NSmF76ShQsXWvW+6HQ6hIWFYdy4cdBqte32Or+WpgLFBejeowcmDY1st+ftaAwGAxITEzF27Fio1WpbN8cmWAPWwIJ1YA0A1sCirXWwnAlprRaFlAMHDqCgoABxcXHStrq6Omzfvh3Lly+HXq+HSqWyOiYwMBD5+flW2/Lz8xEYGNjs62g0Gmg0mkbb1Wp1u35YVErTkByFUmnXH0KL9q5vR8QasAYWrANrALAGFq2tQ1tr16KBs6NHj8aRI0eQmpoq/RkwYABmzJiB1NTURgEFABISErB582arbYmJiUhISGhTw9sDF3MjIiKSrxb1pLi7uyM6Otpqm6urK3x8fKTtM2fOREhICBYvXgwAmDdvHkaMGIE333wTkydPxurVq7F//3589NFH7fQWWo+LuREREclXu684m5WVhdzcXOn+kCFDsGrVKnz00Ufo06cPvvvuO6xbt65R2LGF+nVSbNwQIiIiaqRVU5AbSkpKuuJ9AJg2bRqmTZvW1pdqd7wKMhERkXzZ9bV7LOd7jEYbt4OIiIgaseuQYulJISIiIvmx65CiAK/dQ0REJFd2HVKUnIJMREQkW3YdUixTkNmTQkREJD92HlJMKYUZhYiISH7sOqRwCjIREZF82XVIqR84a+OGEBERUSN2HVI4cJaIiEi+7DqkQBqTwpRCREQkN3YdUurHpBAREZHc2HVIsSw4yynIRERE8mPXIYVXQSYiIpIvuw4pCmngLFMKERGR3Nh5SOFibkRERHJl1yGFA2eJiIjky65DCq+CTEREJF92HVK4mBsREZF82XVIAQfOEhERyZZdhxROQSYiIpIvOw8pplteBZmIiEh+7Dqk8CrIRERE8mXfIYUDZ4mIiGTLzkMKr4JMREQkV3YdUriYGxERkXzZdUjhVZCJiIjky75DCqcgExERyZZdhxSl5d2zJ4WIiEh27DqkcAoyERGRfNl1SOG1e4iIiOTLrkNK/ZgUphQiIiK5sfOQYrplRCEiIpIf+w4p5lsu5kZERCQ/dh1SeBVkIiIi+bLzkGK6ZU8KERGR/Nh1SAF7UoiIiGTLrkOKpSeFiIiI5MeuQ0r9Ym7sSiEiIpIbuw4pXMyNiIhIvuw6pFjWSWFPChERkfy0KKSsWLECsbGx0Gq10Gq1SEhIwK+//trs/itXroRCobD64+Tk1OZGtxfLirOMKERERPLj0JKdQ0NDsWTJEnTt2hVCCHz++eeYMmUKUlJS0Lt37yaP0Wq1SEtLk+5bgoEccAoyERGRfLUopNx6661W91955RWsWLECu3fvbjakKBQKBAYGtr6F1xCvgkxERCRfLQopDdXV1WHNmjWoqKhAQkJCs/uVl5cjIiICRqMRcXFxePXVV5sNNBZ6vR56vV66r9PpAAAGgwEGg6G1TW7EaKwz3xrb9Xk7Gst7Zw1Yg4a39op1YA0A1sCirXVoa/0UooXnOo4cOYKEhARUV1fDzc0Nq1atwqRJk5rcNzk5Genp6YiNjUVpaSneeOMNbN++HceOHUNoaGizr/Hiiy9i0aJFjbavWrUKLi4uLWnuFe2/qMAXGSp08zBiTi9juz0vERERAZWVlbj33ntRWloKrVbb4uNbHFJqamqQlZWF0tJSfPfdd/j444+xbds29OrV66rHGgwG9OzZE9OnT8fLL7/c7H5N9aSEhYWhsLCwVW+yOetSzuPpH44jvpMnvpw9qN2et6MxGAxITEzE2LFjoVarbd0cm2ANWAML1oE1AFgDi7bWQafTwdfXt9UhpcWnexwdHREVFQUA6N+/P/bt24dly5bhww8/vOqxarUa/fr1Q0ZGxhX302g00Gg0TR7fnh8WB5XK9BeFwq4/hBbtXd+OiDVgDSxYB9YAYA0sWluHttauzeukGI1Gq16PK6mrq8ORI0cQFBTU1pdtF5arIHNyDxERkfy0qCdl4cKFmDhxIsLDw1FWVoZVq1YhKSkJGzZsAADMnDkTISEhWLx4MQDgpZdewuDBgxEVFYWSkhK8/vrryMzMxEMPPdT+76QVuJgbERGRfLUopBQUFGDmzJnIzc2Fh4cHYmNjsWHDBowdOxYAkJWVBaWyvnPm0qVLePjhh5GXlwcvLy/0798fu3bt+kPjV64HBXtSiIiIZKtFIeWTTz654uNJSUlW95cuXYqlS5e2uFHXi7SYm22bQURERE2w72v38CrIREREsmXXIYVXQSYiIpIvuw4p4LV7iIiIZMuuQ4qSV0EmIiKSLTsPKaZbjkkhIiKSH7sOKZYpyEZetoeIiEh27DykmG7Zj0JERCQ/9h1SYFnMjTGFiIhIbuw6pHAKMhERkXzZdUjhtXuIiIjky65DCqcgExERyZddhxQFF3MjIiKSLfsOKdK1e2zcECIiImrErkMKB84SERHJl12HFGkxN6YUIiIi2bHzkGK6ZUQhIiKSH/sOKeZbDpwlIiKSH7sOKdIUZGYUIiIi2WFIAcekEBERyZFdhxQFZ/cQERHJFkMKOHCWiIhIjuw7pICne4iIiOTKrkMKF3MjIiKSL7sOKbwKMhERkXzZeUhRXH0nIiIisgm7DimcgkxERCRfdh1SLP0ovAoyERGR/Nh1SFGa3z07UoiIiOTHrkOKZQoyr91DREQkP/YdUriYGxERkWwxpIADZ4mIiOTIrkMKr4JMREQkXwwpYE8KERGRHNl1SLFgRiEiIpIfuw4pSg6cJSIiki27DikKnu4hIiKSLbsOKbwKMhERkXzZdUhpeHlBLuhGREQkL/YdUhpcBZkZhYiISF7sOqQoG4QUjkshIiKSF7sOKQ0yCq+ETEREJDMtCikrVqxAbGwstFottFotEhIS8Ouvv17xmDVr1qBHjx5wcnJCTEwM1q9f36YGtydlg5AiOBGZiIhIVloUUkJDQ7FkyRIcOHAA+/fvx80334wpU6bg2LFjTe6/a9cuTJ8+HbNnz0ZKSgqmTp2KqVOn4ujRo+3S+LbjmBQiIiK5alFIufXWWzFp0iR07doV3bp1wyuvvAI3Nzfs3r27yf2XLVuGCRMm4Omnn0bPnj3x8ssvIy4uDsuXL2+XxreVVU8KQwoREZGsOLT2wLq6OqxZswYVFRVISEhocp/k5GQsWLDAatv48eOxbt26Kz63Xq+HXq+X7ut0OgCAwWCAwWBobZMbqa2trX/Nmho4KFpdjg7NUtP2rG1HwxqwBhasA2sAsAYWba1DW+vX4p/KR44cQUJCAqqrq+Hm5oa1a9eiV69eTe6bl5eHgIAAq20BAQHIy8u74mssXrwYixYtarR948aNcHFxaWmTm1VTB1hKsGHjRjip2u2pO6TExERbN8HmWAPWwIJ1YA0A1sCitXWorKxs0+u2OKR0794dqampKC0txXfffYdZs2Zh27ZtzQaV1li4cKFVD4xOp0NYWBjGjRsHrVbbbq9TUaUH9m4DAIwdOxbuTup2e+6OxGAwIDExEWPHjoVazRqwBvZbA4B1AFgDgDWwaGsdLGdCWqvFIcXR0RFRUVEAgP79+2Pfvn1YtmwZPvzww0b7BgYGIj8/32pbfn4+AgMDr/gaGo0GGo2m0Xa1Wt2uHxZ1rVH6u0rVvs/dEbV3fTsi1oA1sGAdWAOANbBobR3aWrs2r5NiNBqtxo80lJCQgM2bN1ttS0xMbHYMy/XGKchERETy1aKelIULF2LixIkIDw9HWVkZVq1ahaSkJGzYsAEAMHPmTISEhGDx4sUAgHnz5mHEiBF48803MXnyZKxevRr79+/HRx991P7vpBUUVivO2rAhRERE1EiLQkpBQQFmzpyJ3NxceHh4IDY2Fhs2bMDYsWMBAFlZWVAq6ztnhgwZglWrVuH555/Hc889h65du2LdunWIjo5u33fRStZTkJlSiIiI5KRFIeWTTz654uNJSUmNtk2bNg3Tpk1rUaOuF/akEBERyZddX7sHABTmsSgck0JERCQvDCnmW57tISIikhe7DymWlGJkSiEiIpIVuw8p7EkhIiKSJ4YU8y17UoiIiOSFIcWcUphRiIiI5IUhxXzLkEJERCQvDCnmW05BJiIikheGFGl2j23bQURERNYYUsy3HDhLREQkLwwp5ltmFCIiInmx+5ACaXYPUwoREZGc2H1IqR84S0RERHLCkGK+5ZgUIiIieWFI4WJuREREssSQYr5lTwoREZG8MKSYb5lRiIiI5IUhhad7iIiIZMnuQ4oFT/cQERHJi92HFKWlJ8W2zSAiIqLL2H1IsWBPChERkbzYfUjhwFkiIiJ5YkjhsvhERESyxJBivjUyoxAREckKQ4r5lj0pRERE8sKQYk4p7EkhIiKSF4YU863gJGQiIiJZsfuQYsGzPURERPJi9yGFy+ITERHJk92HFEsBuJgbERGRvNh9SLFgSCEiIpIXuw8pCl67h4iISJYYUsy3XCeFiIhIXhhSzLfMKERERPLCkMLF3IiIiGSJIcV8y9M9RERE8mL3IcWCPSlERETyYvchpX4xN6YUIiIiObH7kGIpACMKERGRvNh9SLHgYm5ERETy0qKQsnjxYgwcOBDu7u7w9/fH1KlTkZaWdsVjVq5cCYVCYfXHycmpTY1uTwqFKZwwoxAREclLi0LKtm3bMGfOHOzevRuJiYkwGAwYN24cKioqrnicVqtFbm6u9CczM7NNjW5Pltk97EkhIiKSF4eW7Pzbb79Z3V+5ciX8/f1x4MABDB8+vNnjFAoFAgMDW9fCa8wycJaIiIjkpUUh5XKlpaUAAG9v7yvuV15ejoiICBiNRsTFxeHVV19F7969m91fr9dDr9dL93U6HQDAYDDAYDC0pclWGj6XwVDbrs/dkVjet72+f4A1AFgDC9aBNQBYA4u21qGt9VOIVs69NRqNuO2221BSUoIdO3Y0u19ycjLS09MRGxuL0tJSvPHGG9i+fTuOHTuG0NDQJo958cUXsWjRokbbV61aBRcXl9Y0t1krjitxslSJGVF1GOTHUz5ERETtpbKyEvfeey9KS0uh1WpbfHyrQ8rjjz+OX3/9FTt27Gg2bDTFYDCgZ8+emD59Ol5++eUm92mqJyUsLAyFhYWtepNXasud72zGiRIl/u+O3rijX0i7PXdHYjAYkJiYiLFjx0KtVtu6OTbBGrAGFqwDawCwBhZtrYNOp4Ovr2+rQ0qrTvfMnTsXP//8M7Zv396igAIAarUa/fr1Q0ZGRrP7aDQaaDSaJo+9Vh8WhVJl1x9E4NrWt6NgDVgDC9aBNQBYA4vW1qGttWvR7B4hBObOnYu1a9diy5YtiIyMbPEL1tXV4ciRIwgKCmrxsdeCNG6WZ3qIiIhkpUU9KXPmzMGqVavw448/wt3dHXl5eQAADw8PODs7AwBmzpyJkJAQLF68GADw0ksvYfDgwYiKikJJSQlef/11ZGZm4qGHHmrnt9I69VdBZkohIiKSkxaFlBUrVgAARo4cabX9s88+w/333w8AyMrKglJZ30Fz6dIlPPzww8jLy4OXlxf69++PXbt2oVevXm1reTuRroJs01YQERHR5VoUUv7IGNukpCSr+0uXLsXSpUtb1KjriYu5ERERyZPdX7un/nSPbdtBRERE1hhSLH9hTwoREZGsMKSYb9mTQkREJC8MKeaU0so17YiIiOgasfuQYsGeFCIiInmx+5DCKchERETyxJDC0z1ERESyxJBivuU6KURERPLCkCL1pNi2HURERGSNIcV8y4GzRERE8sKQYr4VHDpLREQkK3YfUix4uoeIiEhe7D6kKDm7h4iISJbsPqRYcEwKERGRvNh9SKm/CjJTChERkZwwpJhvmVGIiIjkhSHFfMsxKURERPLCkGIZOGvbZhAREdFlGFLMtxyTQkREJC8MKeZbZhQiIiJ5sfuQAml2j22bQURERNbsPqRw4CwREZE82X1IsRSAEYWIiEhe7D6kSKd7eL6HiIhIVuw+pNRfBZmIiIjkhCHFfMspyERERPLCkCJdBdm27SAiIiJrDCnmW87uISIikheGFPMtx80SERHJC0OKdO0ephQiIiI5sfuQYpnXw54UIiIiebH7kKLkwFkiIiJZsvuQYsGBs0RERPJi9yGFV0EmIiKSJ7sPKSpzSjEYjbZtCBEREVmx+5CiUZluy6trbdsQIiIismL3IcXJElL0DClERERyYvchxdnBdFvGnhQiIiJZsfuQ4qQyjZgtqzbYuCVERETUEEMKT/cQERHJEkOKOaToeLqHiIhIVloUUhYvXoyBAwfC3d0d/v7+mDp1KtLS0q563Jo1a9CjRw84OTkhJiYG69evb3WD25tlTEpNrRH62jrbNoaIiIgkLQop27Ztw5w5c7B7924kJibCYDBg3LhxqKioaPaYXbt2Yfr06Zg9ezZSUlIwdepUTJ06FUePHm1z49uDpScF4DRkIiIiOXFoyc6//fab1f2VK1fC398fBw4cwPDhw5s8ZtmyZZgwYQKefvppAMDLL7+MxMRELF++HB988EErm91+lArAxVGFypo6lOtr4eOmsXWTiIiICC0MKZcrLS0FAHh7eze7T3JyMhYsWGC1bfz48Vi3bl2zx+j1euj1eum+TqcDABgMBhgM7TcLx/JcbhpTSLlUXo1grWO7PX9HYalDe9a2o2ENWAML1oE1AFgDi7bWoa31a3VIMRqNmD9/PoYOHYro6Ohm98vLy0NAQIDVtoCAAOTl5TV7zOLFi7Fo0aJG2zdu3AgXF5fWNrlZilo9AAU2bduJTA/7vYhPYmKirZtgc6wBa2DBOrAGAGtg0do6VFZWtul1Wx1S5syZg6NHj2LHjh1takBTFi5caNX7otPpEBYWhnHjxkGr1bbb6xgMBiQmJiLA2wP5OTr07tsfY3r6t9vzdxSWOowdOxZqtdrWzbEJ1oA1sGAdWAOANbBoax0sZ0Jaq1UhZe7cufj555+xfft2hIaGXnHfwMBA5OfnW23Lz89HYGBgs8doNBpoNI3HhqjV6mvyYdE6m56z0iDs+sN4rerbkbAGrIEF68AaAKyBRWvr0NbatWh2jxACc+fOxdq1a7FlyxZERkZe9ZiEhARs3rzZaltiYiISEhJa1tJryE1jympc0I2IiEg+WtSTMmfOHKxatQo//vgj3N3dpXElHh4ecHZ2BgDMnDkTISEhWLx4MQBg3rx5GDFiBN58801MnjwZq1evxv79+/HRRx+181tpPXcnUxm4ND4REZF8tKgnZcWKFSgtLcXIkSMRFBQk/fnmm2+kfbKyspCbmyvdHzJkCFatWoWPPvoIffr0wXfffYd169ZdcbDt9eausYQU9qQQERHJRYt6UoS4+syXpKSkRtumTZuGadOmteSlrivL6Z4ynu4hIiKSDbu/dg8AuDmxJ4WIiEhuGFLQYOAsx6QQERHJBkMKGg6cZU8KERGRXDCkoD6kcAoyERGRfDCkoMHAWfakEBERyQZDCgCtuSeltIpjUoiIiOSCIQWAj6vpysfl+lpUG+ps3BoiIiICGFIAmMakqFUKAEBRRY2NW0NEREQAQwoAQKFQwMfVdEHDonK9jVtDREREAEOKxNfddMqnqJw9KURERHLAkGJm6UkpZE8KERGRLDCkmPm4mXtSOCaFiIhIFhhSzHzdOCaFiIhIThhSzCzTkDkmhYiISB4YUsx8zD0pF9mTQkREJAsMKWbSmBT2pBAREckCQ4qZn2VMSgV7UoiIiOSAIcWsYU+KEMLGrSEiIiKGFDNv88DZWqOAropXQyYiIrI1hhQzjYMK7uarIRfylA8REZHNMaQ0YBmXkq+rtnFLiIiIiCGlgVBvFwBAdnGljVtCREREDCkNRJhDSmYRQwoREZGtMaQ0EOFjCilZ7EkhIiKyOYaUBsK8GVKIiIjkgiGlAfakEBERyQdDSgNhXqaQUlJpQGmVwcatISIism8MKQ24ahzga56GzBk+REREtsWQcplwb2cAnOFDRERkawwpl4nwcQUAZBZX2LglRERE9o0h5TJdA9wAAClZJbZtCBERkZ1jSLnMTVF+AIDk00Uw1Blt3BoiIiL7xZBymd7BWvi4OqJcX4uDmZds3RwiIiK7xZByGaVSgWFdfQEAf/5kD77em2XjFhEREdknhpQm3NTVdMrHUCew8IcjSMsrs3GLiIiI7A9DShMmRgfi5h7+0v1D2SW2awwREZGdYkhpgqvGAZ/ePxCPDO8MADicU4L0/DKU62tt3DIiIiL7wZByBdEhHgCAL3dnYezS7Zi/OsXGLSIiIrIfDClXEGMOKRZbThagoKzaRq0hIiKyLwwpVxDh7WJ13yiAX4/k2ag1RERE9qXFIWX79u249dZbERwcDIVCgXXr1l1x/6SkJCgUikZ/8vLk/8NeqVQ02vbz4Qs2aAkREZH9aXFIqaioQJ8+ffDee++16Li0tDTk5uZKf/z9/a9+kAwsvacP/Nw1eGd6PwDAvnOXUKDjKR8iIqJrzaGlB0ycOBETJ05s8Qv5+/vD09OzxcfZ2u39QnF7v1AAwCc7zuJQdgm2phXgnoHhNm4ZERHRja3FIaW1+vbtC71ej+joaLz44osYOnRos/vq9Xro9Xrpvk6nAwAYDAYYDIZ2a5Pluf7oc47o6oND2SXYdDwfd/QNard22FpL63AjYg1YAwvWgTUAWAOLttahrfVTCCFEqw9WKLB27VpMnTq12X3S0tKQlJSEAQMGQK/X4+OPP8YXX3yBPXv2IC4ursljXnzxRSxatKjR9lWrVsHFxaWJI66P7HLgjSMOcFQKLB5YB6UCUABQNB66QkREZPcqKytx7733orS0FFqttsXHX/OQ0pQRI0YgPDwcX3zxRZOPN9WTEhYWhsLCwla9yeYYDAYkJiZi7NixUKvVV93faBQY9vo2XCyvwTv3xOKNxHR4OKvx3SPxTQ6y7ShaWocbEWvAGliwDqwBwBpYtLUOOp0Ovr6+rQ4p1+10T0ODBg3Cjh07mn1co9FAo9E02q5Wq6/Jh6UlzzshOghf7M7EwrXHUFFTB6AKKTllGNzZp93bdb1dq/p2JKwBa2DBOrAGAGtg0do6tLV2NlknJTU1FUFBHXNMx4PDIqFQwBxQTH5M5bRkIiKi9tbinpTy8nJkZGRI98+ePYvU1FR4e3sjPDwcCxcuRE5ODv773/8CAN5++21ERkaid+/eqK6uxscff4wtW7Zg48aN7fcurqNIX1dM6B2IX4/mQa1SwFAnsP5ILhbd1huODlwbj4iIqL20OKTs378fo0aNku4vWLAAADBr1iysXLkSubm5yMrKkh6vqanBU089hZycHLi4uCA2NhabNm2yeo6O5qlx3ZF9qRL3D4nEa7+dREGZHjszCjGqR8dY+4WIiKgjaHFIGTlyJK401nblypVW95955hk888wzLW6YnEX5u+Hnv9wEADiQeQlf783CtlMXMaqHP/536AIOZZdg4cQecFCxZ4WIiKi1+FO0jUZ08wUA/J5+EdWGOvz9+8P4ZMdZJKVdtHHLiIiIOjaGlDZK6OILlVKB0xcr8NWeLFSaB9QmnymyccuIiIg6NoaUNvJwVqNvmCcA4OWfj0vbdzOkEBERtQlDSju4uYkBs8dzdSiprLFBa4iIiG4MDCntYPawSEwfFA6FAogO0aKLnyuEAN7elI7s4kppv22nLvIKykRERH+QTVacvdE4qVVYfEcMnhzbFc5qFd7YkIbTFyuwctc5rNmfjWV/6gcB4OH/7seIbn74/MFBtm4yERGR7DGktCN/dycAwF9Gd4XWWY3tpy7i0PlSPPblAcRFeAEwjVX536EL+O7AeSyc2AM9g9rvWkREREQ3Ep7uuQZ83TR4alx3fPf4EAzq5I1ao8Des8UAAH2tEfNXp2D7qYuYuOx3HM0ptXFriYiI5Ikh5RpSq5S4LyGi0XZjg7Xw3ko8ZfVYdnElB9wSERGBIeWaG9srAB7OpqtAKhT1231cHQEAKVmXpBV8950rxs1vJuHe/+wBANQZBZZvSed0ZiIisksMKdeYk1qFWUM6AQAeGBIpbf/r6K5wVClxqdKAzKJKnCuswONfHoChTuB4rg7ZxZX4ak8m3th4Cn/6aLeNWk9ERGQ7HDh7Hcwf3RX3DY6Ai6MK3+zLgsEoMDEmEGtTcpCaXYJpHybjYpne6pidGYXYerJAul9tqIOTWnW9m05ERGQzDCnXgVKpgJ+7BgDw9SODUWcU8Hd3Qt8wT6Rml0gBZWiUD9w0DthwLB87TxfhZF6Z9ByZRZXoHuhuk/YTERHZAk/3XGexoZ7oF26ajtwv3FPaPjk2CF89NBgPDjWdEvrp0AXkltYv/JZRUA4A2Hu2GKVVhuvXYCIiIhthSLGhOHNYAYAnx3QDAPQL94KTuvE/S0ZBOTYey8PdHybjgc/2SoNtiYiIblQMKTYU5u2CZX/qi49nDkCUvxsAwNFBiXmju0HrZDoT18N8iifjYjlW7c0CABzMKsH6I3m2aTQREdF1wjEpNjalb0ijbY+P7IJHh3eGrtqAfecu4eH/7kfy6SIUV9QPrn1+3RF8uTsTB7MuIdDDCXcPCMOcUVGo0NfiwZX70CPQHYumRF/Pt0JERNSu2JMiU0qlAp4ujlIPS2G5HkZhuoBhZ19XXKo0IPlMEfS1RmQWVeKNjWko0FVj04l87DlbjM+TM7E1reAqr0JERCRf7EmRuTAvZzirVagy1AEAHh3eBaN7+iP5dBEulFajf7gXnvn+EI7m6LDpRAH2ZxZLx77883G4qFV4bu0RTIgOxNPje9jqbRAREbUYQ4rMOaiUeOvuPkjNLsHNPfwR39kHADC6Z4C0z8ToIBzN0eG3Y3k4fqH+WkBnLlbgHvNCcO9tPY2Hb+oMTxfH6/sGiIiIWomnezqAiTFBWDippxRQLje+tymwbD91EYXlNXB1VOH7x4cgUOtktd9Xe7Kwak8WnvnuEHZlFF7zdhMREbUFe1JuAFH+7ojyd5PWUhka5Yv+EV745a/D8Ht6IbKKK/FW4im8viFNOmbTiQJs/dtI6bpCREREcsOelBvEsj/1xayECEyKCcR885orPm4aTO0Xgnvjw+GgNF3dMNzbBaFeziiuqME7m9NhNAr8eCgXK44r8dqGU1d6CSIikpniihpkFJRdfccOij0pN4jewR5YNMWjycd83TR4f0YcsoorMSM+AnvPFWPWp3uxctc5ZBZVYtOJfABKnNxxDtMGhsNZrUKIpzOUSkWTz0dERPLw0Of7cPh8KX6bfxOi/G+8S6cwpNiJcb0Dpb+P6OaH2/uFYG1Kjjmg1Hvgs33IKanCX0d3xYKx3WA0iibDSp1R4KlvU5FbWo074kJw94AwHMwqQVJaAZQKBf46uitUDDlERNfMxTI9DmaVAACS0i4ypNCN45Xbo3H8gg5p+WWYO7IzavLS8dFJFXJKqgAA/9l+BilZl3C2sAIrHxiEKH83FJRVo6bWiFAvF6xNycG61AsAgD1ni5Gv02PZ5nTUGU3L9fcJ88DNPQKafX0iInv27f5sBHs4Y1hX3yYfP3+pEv9YexQKBfDhff2hcVBh26mLcFarMCjSGwCw52yRtP++c8UY1tUXvm4a+Lppmn3dc4UVcHRQItjTuX3f0DXCkGKnXBwd8MMTQ3AyrwwxQa74ef0pBGo1yNPp4ahSospQh9/TTTOAHvp8H6b0DcEH205DqVBg3ZyheGujaRBuJx8XnCuqxNJNp9DwckKHsktx+HwpegRqMSE60Oq1q2rqkJJ9CQmdfaBQsLeFiOzLtlMX8cx3hwEAZ16dBIUC2HAsD1383NA1wB3ZxZW4bfkOXKo0XUz2h4M56B7ojlmf7oXGQYm9z42Bh4sau8/Uh5QNx/Kx8Xg+XB0dsOi23rizfygAoNpQh6WbTmFXRhEW3xGD6R/tRq1R4OA/x8LZUXX933wLMaTYMVeNA/pHeMFgMEClAJZP74tDOWUI83LGI18cgMZBCS8XR5wrqsSyzenScfd8lIySSgOCPJzw8awBGPPWdimg9A3zRGp2CVbuOiddrXnHs6NQbahDFz83KBQK/PmTPTiQeQkrZsRhYkyQLd46EZHN/JiaI/39/KUq/O9QDt7YeAo9At3x2/zh+HTnWSmgAMCKpNPwcjWtcaWvNWLD8TzcPSAMu88UWz2vEEC5vhZPrTkEXbUBPx26gJN5ZaisMS0GuvjXEyjT1wIAEk/k47Y+wdKx1YY6OKnlF1o4u4ckfUI9MHtYJMb1DsQHf47Dd48NwTePDsaM+HCM7RWAJ0Z2AQCUmP/zvDwlGlH+7hjUydT16K5xwPwxXQFACigAMOHt3zHmre34ak8WTubpcCDzEgAg8bj1eBgiIjmprgWe//EY9p41hYHKmlos35IunRZvjaqaOmw8Vv/d993B83hjo2lm5cm8MpRU1uD7A+cBACtmxMHb1RFZxZU4lF0iHfPL4VycLaxARkE5Lu+MntrXFDwW/XQcB7NKpIACADsz6nte1h48D6NRQAiBpYmncNcHu1BWbYDcsCeFmjQhur6H45XbY6S/55RU4cfUC5g3uivG9DKNOZk5xDRj6J6BYYiL8Gr0XOXm5P7f5HPYd64++dc1PD9ERCQz351VYl9hDr7Zn4NzSybjw21nsGxzOo7n6vD+jP5XPV4IgeQzRTh+QYdxvQIR7uOCX47kSt+JALB8S7rVMW9vSoeuuhahXs4Y3zsQBqPAu5vTUVNnxC2xQXhv62nszCjEX74+CAAYFuWLEd388O9fTuDJMd3w8PBIHMwqQVZxJTr7uWLFjP7I11Vj5qd7rV5na9pFdH5uPTr7ueLMxQoAwMZj+dJpIrlgSKEW+b87Y/HwTZ3RO1grbbslNhgxIR4I8XSGg0qJCB8XZBZVQqEA/jQwDMcu6HD4fClO5ZfjVH65dFzOJdNvIxuP5eGL3ZlwcVTh8ZFR6Bvm2eRrl1Ub8M91RzG6ZwBubdBNSUR0LRy9ZN1NYblo667TRc3OfNx26iL+/fNxTO0XgsJyPT7beQ6AaVzJD08MwdubTL0mPq6OKKqogfGy39W+2pMJwPTdqVQqcFufYKvTMttOXcTRHB2O5ujgrnHA4jtiEOThjOHd/NDV33RK/bMHBuL7A+dxX0IEgjycEezpBIUCaOr3QktAaTiORU4YUqhFnNQqRIc0Xo8lwsdV+nvvYC0yiyoRG+KBxXfEAgD+8nUKfjpkmg3UK0iL47k6nL9UhTqjwL9+PIY8XTUA4PTFCmyYP7zJ6cvf7j+PdakX8Ht6ISZGB8JBxbOVRHRtGOqMqKqr/x7KKanC4fOma6OVVBpwIk+H3sGm70JdtQFODipsPJ6Hv36dAqMAlibWL47p6KDE8VwdJr/zO85fqkKAVoNXpsbgof/uBwB4OKsxKSYQX+/NhqHOlCRG9fBvsl3v/KkfPt91DqnnS/HXm6MQ6uUCAOgWUD/9uIufG56ZUH9BWXcnNTr7uuK0OZAkPjkcBWV6BGg1SEq7iC7+bhjVvenXszV+y1O7G2O++OHt/UKkbfcOCgcAdPFzxQd/NnWT5pdVY/upi8jTVcPDWQ0PZzUyCsrxv0M5jZ8UkEJOUUUNks2j2o2X/xpCRHZPX1uH9Pwrr8Kqr63DT4cuoKrBmA2Lb/Zl4bGvUqy2fbU70+p+8mnTd9CxC6VIeHUzZn66B29uPAWjAFwdVag1CtQaBQZFemP2sEgAkELC0+N7oF+4p/RcI7v7oWdQfe+0l4saPQPr7zfU2c8Ni6ZE48c5Q60uNHs1saGm1/N316BrgDuGRvkiyt8dD93UWbYBBWBPCl0Dt/cLwZAuvgjQ1s/VT+jig//NHYoIb1donR2gcVBCX2vEu+bzsbf1CUaQpxNe+y0NSxPTEerlgve2ZuCpsd0RE+qB7OJKpDYYOHbfJ3vRLcANpy9WYP7orvjL6K7X+20SkUwt+uk4Vu3Jwlt398EdcU2fwliamI4Ptp3G7GGR+OctvZB8uggbjuXh3vhwLPzhSKPTMKv3ZQMAPF3UKKk04N+/nEBS2kUUlFWjoqZOmmnjrnHAO/f2wwOf7QMAPHJTZ8RFeOG7A+dRVVOHZyZ0x51xIVAoFPBz1+BimR6jewbAz836+7K9V/yOi/DC2pQc9GnmdLpcMaRQu1MoFAj0cGq03ZLkASDE0xlnCiuk1RLv7B+Krv5uWLnzHLKKKzHtg2TTcwH47IFB+OVILoD687gApPEtX+zOxNybo2CoE/jp0AWM6uEPb/N0PSJqvYtleizecBT3DY6QFhBrqcJyPRb9dBz3D4lA/4jWPYcQAofPl6J7oPtVp8mWVRvww0HT7Jhlm9Ox4VgejuboEOXvhrfu7gMfNw3qjELa55fDuQjUOuGV9ScAAD8fzm0UUADTNXIAYO6oKPz7F9O+O5q4mvyd/UMxqrs//nJzFMqqa3FzD38olQpsfmoEHJQKuDjW/9j91y29sO9cMSb0DrSaWTOkS9MLvLXFPQPCoDfUYVyvwKvvLCM83UM2EeJVv9phZz9X9An1gKvGAf+Y3NNqv2MXdACArSdNA9bm3hyF3sFauDqq8PeJpnOuBWV6HM/V4e1Np/DUmkN4+efj0vGF5Xr8Y+0R/M98qqi00oCH/7vfap0CImra2tQL+OnQBbyflNHq51i58xx+OnQBb2784xcwFULgt6N5KDCPVVt/JA9T3ttpdSX35vx6JA/VBiMAILOoEhuO5SOnpArbTl3Eqj1ZAEynagrK9ACAPF21FFAA03cGANwWG4S7O9dh9tAI6TF3jQNmDemEF2/thQVju+GBoZ0QqHXCG9P6IMTTGY4qJe5LMO3/1LjuePG23lKPiNZJbRVQAODWPsF4aUo0HB2U8HHToIufKxwdlBjRze8P1+qPcnRQ4qGbOiPcx6Xdn/taYk8K2URog5ByZ1yotPLsbX2C8b/UC9hsDiUFZXpcKKlCirnHZWR3f9wbHw6VQgEHlRL7zxVj04kC/HQoVxoVv/lEPmrrjDhbWIE73t+FMn0tfjp0AZOiA/Ht/mwkHs9Hen4ZpvQNARE1L93cW3n6YvlV9myeZY2RQ9klqDMKKBVotlfkVH4ZKvS1SC8oxzPfHYavmwafPzgQu06beix2ZhTit6O52JFRiOcn94LGQYmfDuficHYJInxcMDEmCF/vMwWREE9n5JRUwd3JAdP6h+HTnWexLjUHc2+OwromfkkZ2ysA6fllOFdUCQCYmRCOnMPZqPR3k/YZGuULtUqJ+4dGStteuLU3ANM10cqqDejs54bW+mJ2PMqqaxHm3bGCxLXEkEI2YRmRrlAAd8TVhwWFQoGPZg6Aoc6ISct+x5nCCnz8+1nU1BkR4umMTj4uVkvpj+zuj00nCvDBttPSNl11LQ5kXsLKXeek1RV11bVIzS7BxuN5AIDM4kqU62txsUyP+atTcFdcCLQwDcQ1GgXe2ZKOTj6umNpg8O/mE/nILq7ErCGduJw/yZYQAkKg0ZiGsmoDPt91Dnf1D2vydGxT0s3h5PylqlatSFptqJPGklXU1OFUfhl2ZhTi37+cQKSvK96+p680RuJoTinu+mAX9LVG+LubxmcUlusx69O9CDFfZyajoBzPrzuKwvIadPJxxa7TRdhi/oUGAF743zEYBaBWKfDlQ/FYfyQXo7r7I8zbGV/tycTpixVIPlOEXw6bTh/fP6QTVu46B5VSgb9P7IHfjubh9Q1p8HVzREywFjmHgYgGgWFk9+Z7OPzcNfBzb/6aOX9ER7mezvXEkEI2YVlnZVR3fwR5WP/HVCkVUClV6BvmiTOFFfh051kAwJAuja/1M6qHv9X8f183RxSW12BtSg42nzB9eUX5uyGjoBzfH8zBfvNqt0IAJ3N1+GzXORw6X4ojOaUIdlHhX6lbMXtYZ7y9yTSgN9jTGS//fBxPju2KJ746aPoC1TphUhPL+f+YmgMHpRKTY7nUP9mGoc6I25bvhL62Dj//ZZjV6YV3t2Tgo+1nkF1chf+7K9bquA+3ncb5S1V44dZe0tR+o6ifjSIEcLawAm4aB7z883GczCvDhOhAPDfJ+vQsYAombyWewoToQOgNRtTUGaXHDmZdkk65nC2swOzP92PHs6NQrq/Fo18ckE7T5OtMp1wclAoUltegsNw0HqTWKKS/v/ZbGmrqjHB0UOKOfiE4kHkJ6QXlcNc44O0/9UWkryvmjIqSXntMzwD8ciQX81enospQh+4B7nhuUk8Y6oyICfFAFz83/Dk+AkdzSjG2V4AU8jo1OD0y4gohha4NhhSyiRHd/LDqoXj0bmLNFYu+4Z74IaW+W7apq4WGeDpj+fQ4nMzTma8NBMxbnSqNxO8ZpMXsYZH425pD+HpvltWxPx/OxXrzgFyjAM5XKADU4q0G6xv8+ZM9qKk14sGV+6Vtb25Mw/jegVZruWQXV2Le6lQAwIHMSPzzlp5QKBQo0FWjoEzfaG0ZQ50R6muwzkt2cSUCPZyuyXOT/CUez8eJXNM4rlV7svDQTZ2tHgOAwzmlVsdcqqjBkt9OQghgVA8/6erlxXpIoQEA0gvKsXLnWWmw+8e/n8FjI7pApVDg051nkdDFB4M7+2BtSg4+2n4GBzIv4abL/s9+kZyJM4Wm4KN1ckBhuR7fHzyPH1MuIKekCpG+rlIP532DI1BYrsevR/OafK+W8PPylN64Z2A4auuM2HbqIroHuks9tQ3NGtIJvx7Nlcai3JcQAUcHpdWK2h4uaqwwL5FgMJgGsvq5a6TwdvkvVHTt8ZuMbEKhUGBIlC88nNXN7mO6SrLp775ujhjetenfYibHBuGpcd0xtV8Ibu7hj+AGXdl3xoVg+GVflD7mmT8rd52DEKbANLVPEHydGg/pr6k1Ntp2+mIFfj58AUdzSnEwy9Qzsz39ovT4pzvPSleQfvDzfbht+Q6k5dWv2fDBttPo/vyv2NXEzADAdH2QzKKKJh+7knc2p+Om17Ziya8nr7jfvnPFKCirbvHzU+uVVhmu2Zo+m47n440Nafg9/SL+m3xO2v6f389Ia4CcuViOs+ZwkFFQBn1tHT7+/QwmvL0dyzanSz2R3+zLxuq9WThXVIG8Sutey3c3p+NgVglczFfONQrgw+2nccvy37Fsczoe/nw/iitqcPh8CQBTT6VlPMrN5oXJTpr/H0yOCZJ6Of6x9ij2niuGu8YB/5k5AB/e1x93DwjFvDFdEd/MjCLL90JnP1fcaZ5i7KBSYnTPgCYDCgAMivSWAonWycFqHaereWBoJO4bHHH1HandtTikbN++HbfeeiuCg4OhUCiwbt26qx6TlJSEuLg4aDQaREVFYeXKla1oKtmbrgHuWPfEUHzzyGDsePZm6SqgV+LupMZPfxmG6YPCMaq7H6YNCIO/1gl3DwhFuLcLFozthhdu6211zIKx3fD6XTF4vm8degS6N/PMJpYvxP8mZ2LaB8m458Nk5JRU4fdT1oFjz9kinL9UiaM5OhgFpIF/eaXVWPLrSRgF8PGOs02+xtxVKRjxehLe25oB8Qevb/TDwfNSD9CXly061dCujEJM+yAZ9/5nzxWfLymtQApgclSgq8byLelXvCBataEOK3eeRV6pbQPZr0dy0WfRRox4Y6s07dUiPb9M+kHeEnVGgfOXKlFaZcATqw5i+dYM3PfJXuw+UwyFwhTq83V6jHh9K3adLpROfQKAoU7gvo/34t+/nMDJvDKs3HVOemzDsXz8/Ycj+MvXh5B72TX00gtM41Pmj+mKJ8d0AwB8uM10+ggAyvS1eGdzurQqa0VNnfTeHropEm6a+o77qf1C8KdB4VLgcVar8O69/RDl74a4cC+8dlcf+LppMCjSRzrGSV3/4+r5yb2Q0NkHr90Z26KVp6cPCseqh+Lx7WMJcNXwREJH0OJ/pYqKCvTp0wcPPvgg7rjjjqvuf/bsWUyePBmPPfYYvvrqK2zevBkPPfQQgoKCMH78+FY1muxHaxYe8nHTYPEdMVbbXrurj/T3c4X1vRS39wtBnzBPGAwGKBTAIzd1woI1RxDk4YRc8w83pcK0bPXQKF88OqIzvj94XrqSMwB8uy9bCiHTB4Xh673ZOJhZgrAGv9FZBg++0+BiYhfL9CirNiDxeD7ydNWYPSwSxRU10kDA1zekwUmtwuxhkcgursQnO87isRFdmhz0+EmDwKOvNaKoXI/c0mr8bc0hPDepJ4abpzSuMp/yyihofrbGybwy3G9eiCrjlYmyvPzA//2Whu8PnocQaHYhvye/ScWvR/OQfKYIH9434Dq3EKjQ10KtUmL5VtP03eziKjy39ggmxwZB46BCVU0dxi7dDgD4/ZlRTc7oKK004Ku9mSjQ6fH3iT3gpFbh1yO5eGX9CZy/VIW+YZ6oqTXC29URDkoFCsr0uDU2GPcMDMPf1hxCbmk1/vXjMWidrL/q9567cjA6mV+OQifTv3vfME/p89vV3w0PDI1ERkE5lpqvQeOuccArd8Tgr1+nWAUewDSGROOgxMBO3tjy1AhppegxPf2hUCjwn5kDcPh8Ke4eEAoft8aDTrsHukPr5ABddS0mxQThh4M50DgoMSM+XFrFtaWGRLX/GiR07bQ4pEycOBETJ078w/t/8MEHiIyMxJtvvgkA6NmzJ3bs2IGlS5cypJBNhHu7oIufK0qrarFwYg+rx26NDcKwrv7IKanC7e/vMm3rE4y37u4LyxCULn7118AATAtGAaYgc9/gTvh6bzYOnS+xOpWVklWCi2V6rNmfLW07klOKye/sQFaxacqjUqGA6rKBwW9uTMPkmCBMeW8niitqUFxRg17BWpzKL8Nzk3rC102Dcn2tNA7BXeOAMn0tUrJKsOFYHk7mleHL3ZlSSGl4ifnmxsWsP1I/BuBcUQWi/K/cu3Ql+84V40DmJTx8U+cmr8d0NZaepMsHTO89Z/phd8octoQQ0NcapdknZdUGaSzDhmOmsRj5umqsSDqNh4d3lmaLAKbQWlCmR6iX8x+aXWGoM+KDpNOIDvVodjnx85cqcfv7u3DRPP4BMA0CrTYYkZpVgvjOPvj58AXpsQOZl+Dposbz647CUGfEW3f3RZ1R4Jblv0s9Fd0C3DEo0gvzVqdK4zEs4WH2sEg8MbILckqq4O/uBEcHJX6bPxzxr26SAqlapcD43oH42TyzJT7SG1H+bvhqTxYcHZR4/a5YfLrzHMqrDTh9sQKF1aaaz0yIkF5n0ZTeUKuU6BHojm4BbkgvKMc79/bDqO7++HpPlhRCGuoVrIVapYS/1qnRtP+hUb4YeoXQoFIqMCkmCN/uz8ashE6Ij/SGj6umxbOMqOO65v1dycnJGDNmjNW28ePHY/78+c0eo9frodfX/+fW6UxfwAaDQRrM1B4sz9Wez9kR2WMd1j0+GIY6I9ydVFafK4PBAC9nNZwdnKFUmM67Rwe7w1hXC8volPG9AvD+tjNS97NlcOGYnn7o7OMEV40KFfo6/Has/od9VnEl3t18CoY6gb5hHsgsqsSlSoMUUABg9d4sOJu/fF+8tSd+TL2AlOxS3P7+Tmm1y80n86WF6baeLMCaR+KRfakKRgGEejohvrM3vj94AfvPFUk/MI5fKIXBYIAQwqoXKfdSBYIa9MqY9gF+Ng8mBoCj50sQ4eWE/x3KRbi3s3SFaiHEVadhZzZYOTjI3RGTYlq20mWFvhZ3frgHWicHvH13rBQgCsr00g/usxfLYTAY8O7W03gv6Qz++0B/DOrkjVW7z0nPo3FQQq+vwau/HMePh3JRVl2DJbdHo6y6Fv9efxI/pJjq6eKowv+eSIC/mwpCWP9/OH+pCs6OKvi4OuKzXZl4M/EUXDUq/P63EXA391L871AuyvW1uGdAKOavTrEKKLf3C0aNwYhfjuZhZ/pFxIVprU7L7T5TiI9/P4Oj5sULA91PoEegu/Q+ASDxeC6+P5CNmjojhnf1QYW+DgfMg1gn9fZHbW0tAtzUgKiDwVAHFwdgQq8ArDtk+ve8JTYIAyO8pJByz4AQ9A3zQOLxfIzp6YdJvf0xqbc/0gvKMeldU0CfNyoSt8YE4HxxFNydHDAw3EOqy8pZ/VFcUYPuge4wGAyYlRDWZEiJDta26bvl+UndMXdkJAK0TugVaLqQ6fX4rrLH78WmtLUOba2fQvzRk95NHaxQYO3atZg6dWqz+3Tr1g0PPPAAFi5cKG1bv349Jk+ejMrKSjg7N/7N5cUXX8SiRYsabV+1ahVcXLjIDV0fS4+okFkOPB1bh5D6izyjWA98eEKFAX5GGAWw8bwSQwIEJocb4aQClh9TIl1nCjDOKgE3NXCxuv4H+p+j6nCgUIETJaZ9RgQZsTtfAb3RtI9KIfBy/zoU64G3jqhgRPNhYEiAEe5qYMN5JQb4GhGlFVh9RgUfjUCRvv64xQNrUVkLvJxS/3vJE73q4KAQ6OxePxAxswx462j9PmNCjOjlacQ7xxzg6mBq18Vq4P3jKvg6AXdGWtfGwiiAZUdVOFdueuK+Pkbc39WIQ8UKVNYCg/1Ni3pZvn0U5kD4VYYSAsCMKCMOFirwZYYptLmpBZ6OqYOnBkgtUuCzU6btGpXAkoF1ePGACqUGBQb4GnFfVyMWp6qQV1X//hf2qcWbR1SoMSrgoxH4V1wdVmUoseeiEgoIuDgAFbUKeGsESmuAMcECk8JN4bOgCnjtsApaNTAvug6LU1XS1XF9NAJOKqC/rxH/yzK1KcJNILNcAY1SIMzNVNMnY+qQWa7AN2dU6OIucFdkHf7vcH2dFRAQUMBZJayuvAsAsd5GHC6u7/HSKAX+3rcOlbWmGnfzEHi4R+MB3gCQUQq8e9z0Os/2qYVKAbyaarr/Rnwt1M2cydtdoIDBCAwLEPijSwIZBfDkbgfp36vcYDrwz1F1GOjHC4Haq8rKStx7770oLS2FVtv0RROvRJYjhxYuXIgFCxZI93U6HcLCwjBu3LhWvcnmGAwGJCYmYuzYsVCrm59lcqNjHZquwYCb9CjQ6REd0vgz9+fbTbdNnY44pclAetIZAMDr0/pi66mL+P6g6bd1b1c1/j5jDF5ZfxIn9pkGUf7rT8Px0e9n8c1+03Trf07uiWnxpqtG944rwlPfHUa5vs5qplGAuwb5ZXoc0zmiq78bgBLcNqQ3BkR4YfW7u6wCCgC8n+GO7EvWIyH/d8EV50uq8debu+Avo7rAYDBgwaebrfapdfXHJTdnANmmH+I947Fq62mUGkpQagDeO6nBBzP64fkfjyPS1wXzR5suW7BqbzbOldcvNZ5apMRKFy+kZpsGVQ6Mi8Honv6Y/cVBFJXr8Z/74nChpBr7d5umet/Upxuyi0sAmMb6lBsUOOfUBc9N7I6U9ScBmMbW6OsUcI0aiNLdpivWplc4osfAeOQl74RKqYCPqyMKyvRIU4ajxmi+irZegfA+w3Bw7x4AAv+5Lw6dfF1xy/JdKNabapyYAyy8ezgKKmqx40AODMYcFOmBj864o6quCk5qJaoNRqnOOVn1px8yyxVQKEzjoCZGB6Cm1giNWoXMokp88/YOZFUqke8aASALoZ5OOF9SDWEOok+O64Gs4kp8uaf+tOBbs4bjrg/2oKTK9BvpX0Z3w5+Hm8ZjTL/VACe1qtkp50IIiC2noXVW44Ehptkp3foWma6E69/8yqhjW/md4NOzCJ/szMSjN0Xi3k9M45r+PGk4uvg1kWRljt+LJm2tg+VMSGtd85ASGBiI/Px8q235+fnQarVN9qIAgEajgUbTeBCVWq2+Jh+Wa/W8HQ3rYF2DEG81QrxbvsT1g8M6o1YAt8YGIzrEA92DPFChN8LNyQF39Q+Fm7MGdw0Iw9f7zmNENz90CfDA3Ju7IbO4CnfGhWLagDDpuUb1DMTvz/iitMqAp749JHWnzx3dFcu3pCNfp8f+zBIAQHxnP3QLcMPwbn7YfuqiVZsuDygAcL7ENDD4nS2n8dDwLnBWq3HWPFN6St9g/Jh6ASfzynE8t3769AOfH4AQptkYYd7OOJVfjke/SkFlTR0yiyux+2wx1jw6BEs3mwaLvnBrL7y39TQKy/VSQAGApZsysCOjCPvOmQYgP/jfgwhvMHD07c3114r51y298NLPx/HN/vO4rW8Iki6bSfXZrvr1b0qravF6ommMUHykN7ROavx2LE86pWPx1HdHYKgT6BfuiTG9gwEA/3dnLD7fdQ4Hs0pghAIjl+5qXLNLVVApFfhk1kDM/yYVF8v00vLrzmoVRnTzw6YT+Xjxtt6YEmf6d3Q0T0rrEqCV9rWEkCfHdsfCH45IY0xu6RMCXzcNDufocPh8KfpHeCEqwAPxnb2lsTUPDe8Ctfm0oPcf+P/6twnWC66N7PHHT7u19DthePdADO8eCCEEJsUEwmgEugV6tPsVfa8nfi+atLYOba3dNQ8pCQkJWL9+vdW2xMREJCQkXOuXJrIJHzcNFk6s/8HQNcAdH9zX32qf/hHe2PjkcGkAZ5i3C1Y/0vT/CVeNA1w1DogN9ZBCyuBIb2T3DcFH2009NgmdfdDV3w0KhQJL7+6Dm9/chtIqAwZ28pKCgEqpwINDO+FCabW0LLjF3R8kY1wvf+n0zL2DwvFj6gXk6ayn71pOzzw1rhv83DWYtzoVlea1ODQOpt6FO1fsQk2dEd0C3HDf4AhkFVfis53n4OKowrePJuChz/cjp6QKOalVUCgAPzcNMosqkWm+ZkqfUA8cMk9j7RWkxQNDO+Hb/dk4mVcmDWZ2d3JAqJcLTuTqpJp4uahxqdKATebptmN6BqCoQg8cq2//mJ6myyhYFhS7f0gn6bEpfUMwpW8Ilm8+hTcS62dhAaYZLucvVaKwvAYLxnbD0ChffPdYAgrLaxDh44J//3wcY3oF4JbYYJTra62m21ooFAo8dFMkFv10HLVGAUcHJcZHB+Lz5HM4fL4UAyK8pHE3X8yOx+e7zmFitClQ/H1iT1QZjJgzskuHGDSqUCjw/oz+V9+R6CpaPLewvLwcqampSE1NBWCaYpyamoqsLNNvMwsXLsTMmTOl/R977DGcOXMGzzzzDE6ePIn3338f3377LZ588sn2eQdEHVS3APcWrdUQE2patdbTRY0ufqYAEOnrij8PDsdnDwyUflv1cdPgt/k34ZNZA3D/kPppml/Ojsc/JvdCr6DGp69O5pXhnS2nTadPNCoM6ORtdRHIW2KDpB+8t8QGYfawSEyIDoSvm6mbYGiUD376yzColArU1BnhrnHAW3f3hYNKib/e3BXPTOiOxAUjEB3igVduj0ZnP1f0CtLi9bv64LvHhsDbvAZOZz9X/PDEULx2VyyGdPHBc5NMK/f+fWIPeLs6wtFBiZu6+mLtE0MwsJOX1D5HlRKL74hBw1/Yx/YKQJcGF3sb3zsA95pPowHAqO5+TV7e4I5+wVArTSHivXvjMH9MV7w3Iw4rHxiE1+6KxeMjugAAInxc0T/CC75uGrz9p364JdbUI9NUQLH48+AIRJlPs4zq7gc3jYO0SOE9A+t70Dyc1fjr6K7oGmCaWRXp64r/PjgI8Z19Gj8p0Q2sxT0p+/fvx6hRo6T7lrEjs2bNwsqVK5GbmysFFgCIjIzEL7/8gieffBLLli1DaGgoPv74Y04/Jmqh0T0CMDkmCMO6+kKpVCDM2wVb/zayyX2DPJwR5OGMonI9PJzV6BHojsGdTat3BmrrZ/T0DNLitTtj8cG20/jFPKunb6gnVEoF/jauO74/eB7dA9zx8PDOuL1fKY5d0OGxEV2gUCigcVDhybHdsGxTOp4a1x3dzNdC+XZfNv59e7R0KQAvV0c8MbL+GiqjewZgdM8Aq/b+Nu8mLN+agckxQVApFbh7QBjubnDaa2R3fxz851irYxqGqEdHdMaE6CCsmzMU3x84j3AfV4R5u0izogBg7qiu6BnkjvsGR6CznytmJXRq8jSEn7sGC2LqMGrEMPQI9gRgCjIhns6NLm/QUmqVEm/d3QdvbjyFeaNNC6L9dXRX3NonGN2vspAgkT1qcUgZOXLkFVfBbGo12ZEjRyIlJaWlL0VEDTg7qvDejLgWHePjpsGe50ZDqVBIg3sDGoSUbgFuiAn1wMJJPaSQYhnkOLVfiNVVoAO0To3CxYz4CMyIr18ufPawyFYtsuWvdcJLU6JbdEyfUE/p75YQFBvqidgG26NDPHBnXChCPJ2knqiXp179dYJdcM0Ge8aGeuLzBwdJ9x0dlAwoRM2Q5eweImo/l49hCNDWD0rvZj6dEOrlgidGdMZnO07jTwNDr2v7WmtQpDc+uq8/okM84OzY9DgNlVKBN+/u0+RjRCR/DClEdsbfqiel/jf4J8dEoZv+1BWnpsqJQqHAuN4tWyCOiDoW+V2Ug4iuKa2TAzxdTNMCewZZn2b4owt3ERFdD+xJIbIzCoUCH/y5P4rKa5q9rD0RkRwwpBDZocGcykpEHQBP9xAREZEsMaQQERGRLDGkEBERkSwxpBAREZEsMaQQERGRLDGkEBERkSwxpBAREZEsMaQQERGRLDGkEBERkSwxpBAREZEsMaQQERGRLDGkEBERkSwxpBAREZEsdYirIAshAAA6na5dn9dgMKCyshI6nQ5qtbpdn7sjYR1YA4A1sGAdWAOANbBoax0sP7ctP8dbqkOElLKyMgBAWFiYjVtCRERELVVWVgYPD48WH6cQrY0315HRaMSFCxfg7u4OhULRbs+r0+kQFhaG7OxsaLXadnvejoZ1YA0A1sCCdWANANbAoq11EEKgrKwMwcHBUCpbPsKkQ/SkKJVKhIaGXrPn12q1dv0htGAdWAOANbBgHVgDgDWwaEsdWtODYsGBs0RERCRLDClEREQkS3YdUjQaDV544QVoNBpbN8WmWAfWAGANLFgH1gBgDSxsXYcOMXCWiIiI7I9d96QQERGRfDGkEBERkSwxpBAREZEsMaQQERGRLNl1SHnvvffQqVMnODk5IT4+Hnv37rV1k1pl8eLFGDhwINzd3eHv74+pU6ciLS3Nap+RI0dCoVBY/Xnssces9snKysLkyZPh4uICf39/PP3006itrbXaJykpCXFxcdBoNIiKisLKlSuv9dv7w1588cVG77FHjx7S49XV1ZgzZw58fHzg5uaGO++8E/n5+VbP0dFr0KlTp0Y1UCgUmDNnDoAb83Owfft23HrrrQgODoZCocC6deusHhdC4F//+heCgoLg7OyMMWPGID093Wqf4uJizJgxA1qtFp6enpg9ezbKy8ut9jl8+DBuuukmODk5ISwsDK+99lqjtqxZswY9evSAk5MTYmJisH79+nZ/v825Uh0MBgOeffZZxMTEwNXVFcHBwZg5cyYuXLhg9RxNfX6WLFlitY+c63C1z8L999/f6P1NmDDBap+O/lm4Wg2a+n5QKBR4/fXXpX1k9TkQdmr16tXC0dFRfPrpp+LYsWPi4YcfFp6eniI/P9/WTWux8ePHi88++0wcPXpUpKamikmTJonw8HBRXl4u7TNixAjx8MMPi9zcXOlPaWmp9Hhtba2Ijo4WY8aMESkpKWL9+vXC19dXLFy4UNrnzJkzwsXFRSxYsEAcP35cvPvuu0KlUonffvvtur7f5rzwwguid+/eVu/x4sWL0uOPPfaYCAsLE5s3bxb79+8XgwcPFkOGDJEevxFqUFBQYPX+ExMTBQCxdetWIcSN+TlYv369+Mc//iF++OEHAUCsXbvW6vElS5YIDw8PsW7dOnHo0CFx2223icjISFFVVSXtM2HCBNGnTx+xe/du8fvvv4uoqCgxffp06fHS0lIREBAgZsyYIY4ePSq+/vpr4ezsLD788ENpn507dwqVSiVee+01cfz4cfH8888LtVotjhw5cs1rIMSV61BSUiLGjBkjvvnmG3Hy5EmRnJwsBg0aJPr372/1HBEREeKll16y+nw0/B6Rex2u9lmYNWuWmDBhgtX7Ky4uttqno38WrlaDhu89NzdXfPrpp0KhUIjTp09L+8jpc2C3IWXQoEFizpw50v26ujoRHBwsFi9ebMNWtY+CggIBQGzbtk3aNmLECDFv3rxmj1m/fr1QKpUiLy9P2rZixQqh1WqFXq8XQgjxzDPPiN69e1sdd88994jx48e37xtopRdeeEH06dOnycdKSkqEWq0Wa9askbadOHFCABDJyclCiBujBpebN2+e6NKlizAajUKIG/9zcPmXstFoFIGBgeL111+XtpWUlAiNRiO+/vprIYQQx48fFwDEvn37pH1+/fVXoVAoRE5OjhBCiPfff194eXlJNRBCiGeffVZ0795dun/33XeLyZMnW7UnPj5ePProo+36Hv+Ipn44XW7v3r0CgMjMzJS2RUREiKVLlzZ7TEeqQ3MhZcqUKc0ec6N9Fv7I52DKlCni5ptvttomp8+BXZ7uqampwYEDBzBmzBhpm1KpxJgxY5CcnGzDlrWP0tJSAIC3t7fV9q+++gq+vr6Ijo7GwoULUVlZKT2WnJyMmJgYBAQESNvGjx8PnU6HY8eOSfs0rJllHznVLD09HcHBwejcuTNmzJiBrKwsAMCBAwdgMBis2t+jRw+Eh4dL7b9RamBRU1ODL7/8Eg8++KDVhTnt4XNgcfbsWeTl5Vm118PDA/Hx8Vb/7p6enhgwYIC0z5gxY6BUKrFnzx5pn+HDh8PR0VHaZ/z48UhLS8OlS5ekfTpKXQDT94RCoYCnp6fV9iVLlsDHxwf9+vXD66+/bnWq70aoQ1JSEvz9/dG9e3c8/vjjKCoqkh6zt89Cfn4+fvnlF8yePbvRY3L5HHSICwy2t8LCQtTV1Vl9EQNAQEAATp48aaNWtQ+j0Yj58+dj6NChiI6Olrbfe++9iIiIQHBwMA4fPoxnn30WaWlp+OGHHwAAeXl5TdbD8tiV9tHpdKiqqoKzs/O1fGtXFR8fj5UrV6J79+7Izc3FokWLcNNNN+Ho0aPIy8uDo6Njoy/kgICAq74/y2NX2kcuNWho3bp1KCkpwf333y9ts4fPQUOWNjfV3obvx9/f3+pxBwcHeHt7W+0TGRnZ6Dksj3l5eTVbF8tzyEl1dTWeffZZTJ8+3eqicX/9618RFxcHb29v7Nq1CwsXLkRubi7eeustAB2/DhMmTMAdd9yByMhInD59Gs899xwmTpyI5ORkqFQqu/ssfP7553B3d8cdd9xhtV1OnwO7DCk3sjlz5uDo0aPYsWOH1fZHHnlE+ntMTAyCgoIwevRonD59Gl26dLnezbwmJk6cKP09NjYW8fHxiIiIwLfffiurH5zXyyeffIKJEyciODhY2mYPnwO6MoPBgLvvvhtCCKxYscLqsQULFkh/j42NhaOjIx599FEsXrz4hlge/k9/+pP095iYGMTGxqJLly5ISkrC6NGjbdgy2/j0008xY8YMODk5WW2X0+fALk/3+Pr6QqVSNZrZkZ+fj8DAQBu1qu3mzp2Ln3/+GVu3bkVoaOgV942PjwcAZGRkAAACAwObrIflsSvto9VqZRkCPD090a1bN2RkZCAwMBA1NTUoKSmx2qfhv/mNVIPMzExs2rQJDz300BX3u9E/B5Y2X+n/emBgIAoKCqwer62tRXFxcbt8NuT0nWIJKJmZmUhMTLTqRWlKfHw8amtrce7cOQA3Th0sOnfuDF9fX6vPv718Fn7//XekpaVd9TsCsO3nwC5DiqOjI/r374/NmzdL24xGIzZv3oyEhAQbtqx1hBCYO3cu1q5diy1btjTqhmtKamoqACAoKAgAkJCQgCNHjlj9B7V8ifXq1Uvap2HNLPvItWbl5eU4ffo0goKC0L9/f6jVaqv2p6WlISsrS2r/jVSDzz77DP7+/pg8efIV97vRPweRkZEIDAy0aq9Op8OePXus/t1LSkpw4MABaZ8tW7bAaDRKIS4hIQHbt2+HwWCQ9klMTET37t3h5eUl7SPnulgCSnp6OjZt2gQfH5+rHpOamgqlUimdArkR6tDQ+fPnUVRUZPX5t4fPAmDqae3fvz/69Olz1X1t+jlo0TDbG8jq1auFRqMRK1euFMePHxePPPKI8PT0tJrV0FE8/vjjwsPDQyQlJVlNGausrBRCCJGRkSFeeuklsX//fnH27Fnx448/is6dO4vhw4dLz2GZejpu3DiRmpoqfvvtN+Hn59fk1NOnn35anDhxQrz33nuymn771FNPiaSkJHH27Fmxc+dOMWbMGOHr6ysKCgqEEKYpyOHh4WLLli1i//79IiEhQSQkJEjH3wg1EMI0Uy08PFw8++yzVttv1M9BWVmZSElJESkpKQKAeOutt0RKSoo0a2XJkiXC09NT/Pjjj+Lw4cNiypQpTU5B7tevn9izZ4/YsWOH6Nq1q9W005KSEhEQECDuu+8+cfToUbF69Wrh4uLSaMqlg4ODeOONN8SJEyfECy+8cF2nIF+pDjU1NeK2224ToaGhIjU11ep7wjJDY9euXWLp0qUiNTVVnD59Wnz55ZfCz89PzJw5s8PU4Uo1KCsrE3/7299EcnKyOHv2rNi0aZOIi4sTXbt2FdXV1dJzdPTPwtX+PwhhmkLs4uIiVqxY0eh4uX0O7DakCCHEu+++K8LDw4Wjo6MYNGiQ2L17t62b1CoAmvzz2WefCSGEyMrKEsOHDxfe3t5Co9GIqKgo8fTTT1utjyGEEOfOnRMTJ04Uzs7OwtfXVzz11FPCYDBY7bN161bRt29f4ejoKDp37iy9hhzcc889IigoSDg6OoqQkBBxzz33iIyMDOnxqqoq8cQTTwgvLy/h4uIibr/9dpGbm2v1HB29BkIIsWHDBgFApKWlWW2/UT8HW7dubfLzP2vWLCGEaRryP//5TxEQECA0Go0YPXp0o9oUFRWJ6dOnCzc3N6HVasUDDzwgysrKrPY5dOiQGDZsmNBoNCIkJEQsWbKkUVu+/fZb0a1bN+Ho6Ch69+4tfvnll2v2vi93pTqcPXu22e8Jyxo6Bw4cEPHx8cLDw0M4OTmJnj17ildffdXqB7gQ8q7DlWpQWVkpxo0bJ/z8/IRarRYRERHi4YcfbvSLaUf/LFzt/4MQQnz44YfC2dlZlJSUNDpebp8DhRBCtKzvhYiIiOjas8sxKURERCR/DClEREQkSwwpREREJEsMKURERCRLDClEREQkSwwpREREJEsMKURERCRLDClEREQkSwwpREREJEsMKURERCRLDClEREQkSwwpREREJEv/D69CACc33jqYAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"][::50]], [i[\"loss\"] for i in record[\"train\"][::50]], label=\"train\")\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 推理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:03:42.243597600Z",
     "start_time": "2024-07-29T08:03:42.228375900Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:19:08.456317Z",
     "iopub.status.busy": "2025-01-23T08:19:08.455886Z",
     "iopub.status.idle": "2025-01-23T08:19:08.467402Z",
     "shell.execute_reply": "2025-01-23T08:19:08.466888Z",
     "shell.execute_reply.started": "2025-01-23T08:19:08.456284Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4502, 0.5498])\n"
     ]
    }
   ],
   "source": [
    "#下面的例子是为了说明temperature\n",
    "my_tensor = torch.tensor([0.4,0.6]) #这里是logits\n",
    "\n",
    "probs1 = F.softmax(my_tensor, dim=-1)\n",
    "print(probs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:17:00.048904100Z",
     "start_time": "2024-07-29T08:17:00.043755800Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:19:08.468453Z",
     "iopub.status.busy": "2025-01-23T08:19:08.468007Z",
     "iopub.status.idle": "2025-01-23T08:19:08.472351Z",
     "shell.execute_reply": "2025-01-23T08:19:08.471801Z",
     "shell.execute_reply.started": "2025-01-23T08:19:08.468420Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4750, 0.5250])\n"
     ]
    }
   ],
   "source": [
    "my_tensor = torch.tensor([0.2,0.3])  #现在 temperature是2\n",
    "\n",
    "probs1 = F.softmax(my_tensor, dim=-1)\n",
    "print(probs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:06:39.649844900Z",
     "start_time": "2024-07-29T08:06:39.642147800Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-23T08:19:08.473444Z",
     "iopub.status.busy": "2025-01-23T08:19:08.473043Z",
     "iopub.status.idle": "2025-01-23T08:19:08.486270Z",
     "shell.execute_reply": "2025-01-23T08:19:08.485745Z",
     "shell.execute_reply.started": "2025-01-23T08:19:08.473414Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "概率分布: tensor([0.1000, 0.4500, 0.3500, 0.1000])\n",
      "抽取的样本索引: tensor([2])\n",
      "每个样本对应的概率: tensor([0.3500])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# 创建一个概率分布，表示每个类别被选中的概率\n",
    "# 这里我们有一个简单的四个类别的概率分布\n",
    "prob_dist = torch.tensor([0.1, 0.45, 0.35, 0.1])\n",
    "\n",
    "# 使用 multinomial 进行抽样\n",
    "# num_samples 表示要抽取的样本数量\n",
    "num_samples = 5\n",
    "\n",
    "# 抽取样本，随机抽样，概率越高，抽到的概率就越高\n",
    "samples = torch.multinomial(prob_dist, 1, replacement=True)\n",
    "\n",
    "print(\"概率分布:\", prob_dist)\n",
    "print(\"抽取的样本索引:\", samples)\n",
    "\n",
    "# 显示每个样本对应的概率\n",
    "print(\"每个样本对应的概率:\", prob_dist[samples])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:51:40.360874200Z",
     "start_time": "2024-07-29T08:51:38.070276100Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-23T08:19:08.487270Z",
     "iopub.status.busy": "2025-01-23T08:19:08.486885Z",
     "iopub.status.idle": "2025-01-23T08:19:09.442765Z",
     "shell.execute_reply": "2025-01-23T08:19:09.442186Z",
     "shell.execute_reply.started": "2025-01-23T08:19:08.487239Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_801/612744915.py:26: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", map_location=\"cpu\"))\n",
      "  0%|          | 0/1000 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All: I"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 1/1000 [00:00<02:11,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " will assure her sine, and the hand of majesty,\n",
      "Who is as free than his friend, and go with thee to thy father's death,\n",
      "Whi"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 124/1000 [00:00<00:01, 644.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ch with his son and a hanging head\n",
      "As thou art so dishonour of mine o'er the sea,\n",
      "Where you are the honour and th"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▎       | 237/1000 [00:00<00:00, 850.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "e more murdered:\n",
      "The son of Clarence we must die the people,\n",
      "I mean the business, but we will mouth to show them "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 350/1000 [00:00<00:00, 955.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for the heels.\n",
      "\n",
      "EDWARD:\n",
      "No, not a word; you must needs mustering years\n",
      "His face and fear and send the stone, for I know how far of"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 480/1000 [00:00<00:00, 1074.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f like the world,\n",
      "And some of these two breathing state,\n",
      "Who having strength, and will not be absent.\n",
      "\n",
      "RICHARD:\n",
      "Brother, no doubt,"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 610/1000 [00:00<00:00, 1147.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " sir, be gone.\n",
      "\n",
      "ROMEO:\n",
      "What says the bastard in the farewell' born in stirr'd with his friends about,\n",
      "And be the present at me,\n",
      "An"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 740/1000 [00:00<00:00, 1196.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d now I protest there were fled,\n",
      "As is the foolish wounds and head is not fourteen;\n",
      "The exchange of his own royal presence shall be"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 871/1000 [00:00<00:00, 1229.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " your penitence\n",
      "With honour, is hither brought\n",
      "The fairest friend,\n",
      "I am the name of some that hath some privilege at the goo"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 995/1000 [00:00<00:00, 1222.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d fro"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:00<00:00, 1061.64it/s]\n"
     ]
    }
   ],
   "source": [
    "def generate_text(model, start_string, max_len=1000, temperature=1.0, stream=True):\n",
    "    input_eval = torch.Tensor([char2idx[char] for char in start_string]).to(dtype=torch.int64, device=device).reshape(1, -1) #bacth_size=1, seq_len长度是多少都可以 （1,5）\n",
    "    hidden = None\n",
    "    text_generated = [] #用来保存生成的文本\n",
    "    model.eval()\n",
    "    pbar = tqdm(range(max_len)) # 进度条\n",
    "    print(start_string, end=\"\")\n",
    "    # no_grad是一个上下文管理器，用于指定在其中的代码块中不需要计算梯度。在这个区域内，不会记录梯度信息，用于在生成文本时不影响模型权重。\n",
    "    with torch.no_grad():\n",
    "        for i in pbar:#控制进度条\n",
    "            logits, hidden = model(input_eval, hidden=hidden)\n",
    "            # 温度采样，较高的温度会增加预测结果的多样性，较低的温度则更加保守。\n",
    "            #取-1的目的是只要最后，拼到原有的输入上\n",
    "            logits = logits[0, -1, :] / temperature\n",
    "            # using multinomial to sampling\n",
    "            probs = F.softmax(logits, dim=-1) #算为概率分布\n",
    "            idx = torch.multinomial(probs, 1).item() #从概率分布中抽取一个样本,取概率较大的那些\n",
    "            input_eval = torch.Tensor([idx]).to(dtype=torch.int64, device=device).reshape(1, -1) #把idx转为tensor\n",
    "            text_generated.append(idx)\n",
    "            if stream:\n",
    "                print(idx2char[idx], end=\"\", flush=True)\n",
    "    return \"\".join([idx2char[i] for i in text_generated])\n",
    "\n",
    "\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", map_location=\"cpu\"))\n",
    "start_string = \"All: \" #这里就是开头，什么都可以\n",
    "res = generate_text(model, start_string, max_len=1000, temperature=0.5, stream=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-23T08:22:23.716047Z",
     "iopub.status.busy": "2025-01-23T08:22:23.715677Z",
     "iopub.status.idle": "2025-01-23T08:22:23.720173Z",
     "shell.execute_reply": "2025-01-23T08:22:23.719579Z",
     "shell.execute_reply.started": "2025-01-23T08:22:23.716020Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"I will assure her sine, and the hand of majesty,\\nWho is as free than his friend, and go with thee to thy father's death,\\nWhich with his son and a hanging head\\nAs thou art so dishonour of mine o'er the sea,\\nWhere you are the honour and the more murdered:\\nThe son of Clarence we must die the people,\\nI mean the business, but we will mouth to show them for the heels.\\n\\nEDWARD:\\nNo, not a word; you must needs mustering years\\nHis face and fear and send the stone, for I know how far off like the world,\\nAnd some of these two breathing state,\\nWho having strength, and will not be absent.\\n\\nRICHARD:\\nBrother, no doubt, sir, be gone.\\n\\nROMEO:\\nWhat says the bastard in the farewell' born in stirr'd with his friends about,\\nAnd be the present at me,\\nAnd now I protest there were fled,\\nAs is the foolish wounds and head is not fourteen;\\nThe exchange of his own royal presence shall be your penitence\\nWith honour, is hither brought\\nThe fairest friend,\\nI am the name of some that hath some privilege at the good fro\""
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
